# -*- coding: utf-8 -*-
# Python 2.7
from pypatnlp import *
import json
import os
import codecs

DATA_PATH = os.path.join('tasks', 'autorituvastus', 'data')
JSON_PATH = os.path.join(DATA_PATH, 'arvamus.json')

# read in individal authors from json encoded file
authors = {}

f = open(JSON_PATH, 'rb')
line = f.readline()
while line != '':
    article = json.loads(line)
    author = article['author']
    url = article['url'].replace('print/', '')
    if url.endswith('/'):
        url = url[:-1]
    article['url'] = url
    if author not in authors:
        authors[author] = []
    authors[author].append(article)
    line = f.readline()

# convert articles to PyCorpus format and leave only authors with more than 2 articles
for author, articles in authors.iteritems():
    if len(articles) > 2:
        print author, len(articles)
        pycorp_path = os.path.join(DATA_PATH, author.lower().replace(' ', '_') + '.pycorp')
        pycorp = PyCorpus(pycorp_path, autocommit=False)
        for article in articles:
            doc = as_t3doc(parse_plain_doc_from_string(article['content']))
            pycorp[article['url']] = doc
        pycorp.commit()




